#include "WebPageQuery.h"
#include "WebPage.h"
#include <sstream>
#include <fstream>
#include <map>
#include <cmath>
#include "json.hpp"
#include <iostream>

using json = nlohmann::json;
using std::cout;
using std::endl;
using std::istringstream;
using std::ifstream;
using std::make_pair;
using std::multimap;
using std::cerr;

WebPageQuery::WebPageQuery(){
    loadLibrary();

    // const char*const DICT_PATH="../include/cppjieba/dict/jieba.dict.utf8";
    // const char*const HMM_PATH="../include/cppjieba/dict/hmm_model.utf8";
    // const char*const USER_DICT_PATH= "../include/cppjieba/dict/user.dict.utf8";
    // const char*const IDF_PATH="../include/cppjieba/dict/idf.utf8";
    // const char*const STOP_WORD_PATH="../include/cppjieba/dict/stop_words.utf8";

    // static cppjieba::Jieba jieba(DICT_PATH,HMM_PATH,USER_DICT_PATH,IDF_PATH,STOP_WORD_PATH);
    // //cppjieba::Jieba* pJieba(DICT_PATH,HMM_PATH,USER_DICT_PATH,IDF_PATH,STOP_WORD_PATH);
    // _pJieba = &jieba;
}

WebPageQuery::~WebPageQuery(){}

//接收查询词条，执行查询，返回结果
string WebPageQuery::doQuery(const string &str){

    //忽略停用词，并将查询词及其出现次数存入unordered_map<string, int>
    unordered_map<string, int> queryWords;
    istringstream iss(str);
    string temp;
    while(!iss.eof()){
        iss>>temp;
        //是停用词，跳过
        if(_stopWordList.find(temp) != _stopWordList.end()){
            continue;
        }
        //不是停用词，存入容器
        else{
            auto it = queryWords.find(temp);
            if( it != queryWords.end()){
                it->second++;
            }
            else{
                queryWords.insert(make_pair(temp, 1));
            }
        }
    }
    cout<<"5.查询词已装入容器"<<endl;

    //test
    // for(auto testIt = queryWords.begin(); testIt != queryWords.end(); testIt++){
    //     cout<<"查询词："<<testIt->first<<"  ";
    // }
    // cout<<endl;


    unordered_map<int, vector<double>> resultVec;

    if(executeQuery(queryWords, resultVec)){

        vector<double> Base = getQueryWordsWeightVector(queryWords);
        multimap<double, int> RecommWebPage;  //推荐网页的集合，double为余弦相似度，int表示id,

        for(auto it = resultVec.begin(); it != resultVec.end(); ++it){
            //unordered_map<int, vector<double>> &resultVec
            //it2指向查询词的vector<double>
            //X * Y = (x1 * y1 + x2 * y2 + x3 * y3)   cosθ = (X * Y) / (|X|*|Y|)
            double XMultiY = 0, xSquare = 0, ySquare = 0;
            auto it1 = Base.begin();
            auto it2 = it->second.begin();
            for(; it1 != Base.end(); ++it1, ++it2){
                XMultiY += (*it1) * (*it2); 
                xSquare += (*it1) * (*it1);
                ySquare += (*it2) * (*it2);
            }
            double cos = XMultiY / (sqrt(xSquare) * sqrt(ySquare));

            RecommWebPage.insert(make_pair(cos, it->first));
        }
        cout<<"8.推荐网页已排序"<<endl;

        // //test
        // for(auto testRec = RecommWebPage.begin(); testRec != RecommWebPage.end(); ++testRec){
        //     cout<<"cos: "<<testRec->first<<"  id: "<<testRec->second<<endl;
        // }


        vector<int> docIdVec;
        if(RecommWebPage.size() > 5){
            int number = 5;
            auto rit = RecommWebPage.rbegin();
            while(number--){
                docIdVec.push_back(rit->second);
                rit++;
            }
        }
        else{
            for(auto rit = RecommWebPage.rbegin(); rit != RecommWebPage.rend(); ++rit){
                docIdVec.push_back(rit->second);
            }
        }

        //test
        // cout<<"docIdVec.size: "<<docIdVec.size()<<endl;

        // for(auto testDoc = docIdVec.begin(); testDoc != docIdVec.end(); testDoc++){
        //     cout<<"docIdvec: "<<*testDoc<<endl;
        // }

        // for(auto rit = RecommWebPage.rbegin(); rit != RecommWebPage.rend(); ++rit){
        //     docIdVec.push_back(rit->second);
        //     //cout<<"cos: "<<rit->first<<endl;
        // }


        return createJson(docIdVec, queryWords);
    }

    return "";    //或者返回错误信息
}

//加载库文件，数据成员（库）赋值
void WebPageQuery::loadLibrary(){

    //偏移库  unordered_map<int, pair<int, int>> _offsetLib;
    ifstream ifs1("offset.lib");
    if(!ifs1){
        cerr<<"ifs open file "<<"offset.lib"<<" error"<<endl;
        return;
    }

    string line;
    string strId, strStart, strLength;
    int id, start, length;
    while(getline(ifs1, line)){
        istringstream iss(line);
        iss>>strId>>strStart>>strLength;
        id = stoi(strId);
        start = stoi(strStart);
        length = stoi(strLength);
        _offsetLib.insert(make_pair(id, make_pair(start, length)));
    }

    ifs1.close();
    cout<<"1.偏移库读取完毕"<<endl;
    
    //网页库  unordered_map<int, WebPage> _pageLib; 
    //偏移库  unordered_map<int, pair<int, int>> _offsetLib;
    ifstream ifs2("tipepage.lib");
    if(!ifs2){
        cerr<<"ifs open file "<<"tipepage.lib"<<" error"<<endl;
        return;
    }

    int testCount = 0;

    for(auto it = _offsetLib.begin(); it != _offsetLib.end(); ++it){
        ifs2.seekg(0);
        ifs2.seekg(it->second.first);
        string strDoc;
        strDoc.resize(it->second.second);
        ifs2.read(&strDoc[0], it->second.second);

        WebPage page(strDoc);

        _pageLib.insert(make_pair(page.getDocId(), page));

    }
    cout<<"2.网页库读取完毕"<<endl;

    //cout<<"读取文档数："<<_pageLib.size()<<endl;

    ifs2.close();

    //倒排索引表  unordered_map<string, vector<pair<int, double>>> _invertIndexTable;
    ifstream ifs3("invertIndex.lib");
    if(!ifs3){
        cerr<<"ifs open file "<<"invertIndex.lib"<<" error"<<endl;
        return;
    }

    string word;
    string strFreq;
    double freq;
    while(getline(ifs3, line)){
        istringstream iss(line);
        iss>>word;
        vector<pair<int, double>> idAndFreq;

        while(iss>>strId>>strFreq, !iss.eof()){
            freq = stod(strFreq);
            id = stoi(strId);
            idAndFreq.push_back(make_pair(id, freq));
        }

        _invertIndexTable.insert(make_pair(word, idAndFreq));
    }
    cout<<"3.倒排索引表读取完毕"<<endl;

    //test
    // auto testIt = _invertIndexTable.find("考研");
    // for(auto it = testIt->second.begin(); it != testIt->second.end(); it++){
    //     cout<<testIt->first<<"   "<<it->first<<"  "<<it->second<<endl;
    // }


    ifs3.close();
    

    //停用词集 set<string> _stopWordList;
    ifstream ifs4("stop_words_eng.txt");
    if(!ifs4){
        cerr<<"ifs open file "<<"stop_words_eng.txt"<<" error"<<endl;
        return;
    }

    while(getline(ifs4, line)){
        istringstream iss(line);
        iss>>word;
        _stopWordList.insert(word);
    }

    ifs4.close();

    ifstream ifs5("stop_words_zh.txt");
    if(!ifs5){
        cerr<<"ifs open file "<<"stop_words_zh.txt"<<" error"<<endl;
        return;
    }

    while(getline(ifs5, line)){
        istringstream iss(line);
        iss>>word;
        _stopWordList.insert(word);
    }
    cout<<"4.停用词集读取完毕"<<endl;
    ifs5.close();

}

//计算查询词的权重向量
vector<double> WebPageQuery::getQueryWordsWeightVector
        (unordered_map<string, int> queryWords){
    //TF : Term Frequency, 某个词在文章中出现的次数；
    //DF: Document Frequency, 某个词在所有文章中出现的次数，即包含该词语的文档数量；
    //IDF: Inverse Document Frequency, 逆文档频率，表示该词对于该篇文章的重要性的一个系数，其计算公式为：
    //IDF = log2(N/(DF+1))，其中 N 表示文档的总数或网页库的文档数 最后，词语的权重 w 则为：w = TF * IDF
    vector<double> Base;
    for(auto it = queryWords.begin(); it != queryWords.end(); ++it){
        double TF = it->second;
        double DF, IDF, N;
        //倒排索引表  unordered_map<string, vector<pair<int, double>>> _invertIndexTable;
        DF = _invertIndexTable.find(it->first)->second.size();
        N = _pageLib.size();
        IDF = (log(N/(DF + 1)))/(log(2.0)); //换底公式
        Base.push_back(TF * IDF);
    }
    cout<<"7.基准向量计算完毕"<<endl;

    //test
    // cout<<"Base: (";
    // for(auto testBase = Base.begin(); testBase != Base.end(); ++testBase){
    //     cout<<*testBase<<", ";
    // }
    // cout<<")"<<endl;

    return Base;
}

//执行查询
bool WebPageQuery::executeQuery(const unordered_map<string, int> &queryWords,
                    unordered_map<int, vector<double>> &resultVec){
    for(auto &word: queryWords){
        if(_invertIndexTable.find(word.first) == _invertIndexTable.end()){
            //查询词不在倒排索引表中
            cout<<"6.查询词不在倒排索引表中"<<endl;
            return false;
        }
    }

    //倒排索引表 unordered_map<string, vector<pair<int, double>>> 某个查询词在各个文章中的频率
    
    //通过倒排索引表去查找包含所有关键字的文章

    //遍历每个查询词的倒排索引表，存入vector<pair<int, vector<double>>> resultVec
    //int表示id，vector<double>依次表示每个查询词的频率  
    //然后删除vector<double>的size小于查询词个数的元素 

    //先遍历第一个查询词的频率，初始化resultVec
    auto word = queryWords.begin();
    auto it= _invertIndexTable.find(word->first);
    vector<double> vecTemp;
    for(auto IdAndFreq = it->second.begin(); IdAndFreq != it->second.end(); ++IdAndFreq){
        resultVec.insert(make_pair(IdAndFreq->first, vecTemp));
    }

    //重新遍历每个查询词，填入频率
    for(; word != queryWords.end(); ++word){
        it= _invertIndexTable.find(word->first);

        for(auto IdAndFreq = it->second.begin(); IdAndFreq != it->second.end(); ++IdAndFreq){
            //IdAndFreq指向查询词的vector<pair<int, double>>
            //unordered_map<int, vector<double>> resultVec
            auto temp = resultVec.find(IdAndFreq->first);
            if(temp != resultVec.end()){  
                temp->second.push_back(IdAndFreq->second);
            }
        }
    }

    for(auto it = resultVec.begin(); it != resultVec.end();){
        if(it->second.size() < queryWords.size()){
            it = resultVec.erase(it);
        }
        else{
            ++it;
        }
    }

    if(resultVec.size() == 0){
        cout<<"6.不存在同时包含所有查询词的文档"<<endl;
        return false;
    }
    
    cout<<"6.推荐网页已装入resultVec容器"<<endl;

    //test
    // for(auto testVec = resultVec.begin(); testVec != resultVec.end(); testVec++){
    //     cout<<"查询到id："<<testVec->first<<"  ";
    // }
    // cout<<endl;

    return true;       
}


string WebPageQuery::createJson(vector<int> &docIdVec, unordered_map<string, int> &queryWords){
    //网页库  unordered_map<int, WebPage> _pageLib;
    json j;
    for(auto it = docIdVec.begin(); it!= docIdVec.end(); ++it){
        WebPage page = _pageLib.find(*it)->second;

        json jpage;
        jpage["id"] = page.getDocId();
        jpage["title"] = page.getTitle();
        jpage["URL"] = page.getUrl();

        string content = page.getContent();        

        jpage["summary"] = generateSummary(content, queryWords);


        j.push_back(jpage);

        //test
        //cout<<"id: "<<page.getDocId()<<"   title: "<<page.getTitle()<<"   URL: "<<page.getUrl()<<endl;
        //cout<<"summary: "<<generateSummary(content, queryWords)<<endl;

    }

    cout<<"9.Json字符串已生成"<<endl;

    return j.dump();
}

string WebPageQuery::generateSummary(string &content, unordered_map<string, int> &queryWords){
    string summary, strTmp;
    for(auto it = queryWords.begin(); it != queryWords.end(); it++){
        if(summary.find(it->first) != string::npos){
            continue;
        }
        size_t pos = content.find(it->first);
        if (pos != string::npos){
            // 句子开头的标点符号位置
            size_t start1 = content.rfind("。", pos);
            size_t start2 = content.rfind("，", pos);
            size_t start = start1 > start2 ? start1 : start2;

            // 句子结尾的标点符号位置
            size_t end1 = content.find("。", pos);
            size_t end2 = content.find("，", pos);
            size_t end = end1 < end2 ? end1 : end2;

            strTmp = content.substr(start + 3 , end - start - 3);
        }

        //处理截断处的字符
        if(nBytesCode(strTmp[0]) != 3){
            if(nBytesCode(strTmp[1]) == 3){
                strTmp.erase(0, 1);
            }
            else{
                strTmp.erase(0, 2);
            }
        }

        size_t lastIdx = strTmp.size() - 1;
        if(nBytesCode(strTmp[lastIdx - 2]) != 3){
            if(nBytesCode(strTmp[lastIdx]  == 3)){
                strTmp.erase(lastIdx, 1);
            }
            else{
                strTmp.erase(lastIdx - 1, 2);
            }
        }
        summary += strTmp + "......";
    }
    string highlightedText = summary;
    string highcolor="\033[31m";
    string resetcolor="\033[0m";
    for(auto it = queryWords.begin(); it != queryWords.end(); it++){
        size_t pos=highlightedText.find(it->first);
        while(pos!=std::string::npos){
            highlightedText.insert(pos, highcolor);
            highlightedText.insert(pos + it->first.size() + highcolor.size(), resetcolor);
            pos = highlightedText.find(it->first, pos + it->first.size() + highcolor.size() + resetcolor.size());
        }
   }


    return highlightedText;
}

size_t WebPageQuery::nBytesCode(const char ch){
    if(ch & (1 << 7)){
        int nBytes = 1;
        for(int idx = 0; idx != 6; ++idx){
            if(ch & (1 << (6 - idx))){
                ++nBytes;	
            }
            else{
                break;
            }
        }
        return nBytes;
    }
    return 1;
}


WebPageQuery *WebPageQuery::getInstance(){
    if(_pInstance == NULL){
        _pInstance = new WebPageQuery();
    }
    return _pInstance;
}

WebPageQuery * WebPageQuery::_pInstance = NULL;
WebPageQuery::AutoRelease WebPageQuery::_ar;